Imports¶

In [ ]:
import glob
import cv2
import os

import numpy as np 
import pandas as pd 
import tensorflow as tf
import matplotlib.pyplot as plt
import seaborn as sns


from collections import Counter
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import f1_score, roc_auc_score, cohen_kappa_score, precision_score, recall_score, accuracy_score, confusion_matrix
from tensorflow.keras.utils import to_categorical

%matplotlib inline
In [ ]:
print(os.listdir("../input/intel-mobileodt-cervical-cancer-screening"))
['additional_Type_2_v2', 'test_stg2.7z', 'fixed_labels_v2.csv', 'train', 'sample_submission.csv', 'test', 'additional_Type_1_v2', 'sample_submission_stg2.csv', 'test_stg2_7z_password.txt', 'additional_Type_3_v2', 'removed_files.csv', 'solution_stg1_release.csv']

Data import¶

In [ ]:
#getting the total number of images in the training set

base_dir = '../input/intel-mobileodt-cervical-cancer-screening'

train_dir = os.path.join(base_dir,'train', 'train')

type1_dir = os.path.join(base_dir,'Type_1')
type2_dir = os.path.join(base_dir,'Type_2')
type3_dir = os.path.join(base_dir,'Type_3')

type1_files = glob.glob(type1_dir+'/*.jpg')
type2_files = glob.glob(type2_dir+'/*.jpg')
type3_files = glob.glob(type3_dir+'/*.jpg')

added_type1_files  =  glob.glob(os.path.join(base_dir, "additional_Type_1_v2", "Type_1")+'/*.jpg')
added_type2_files  =  glob.glob(os.path.join(base_dir, "additional_Type_2_v2", "Type_2")+'/*.jpg')
added_type3_files  =  glob.glob(os.path.join(base_dir, "additional_Type_3_v2", "Type_3")+'/*.jpg')

type1_files = type1_files + added_type1_files
type2_files = type2_files + added_type2_files
type3_files = type3_files + added_type3_files


print('Number of images in a train set of type 1: ', len(type1_files))
print('Number of images in a train set of type 2: ', len(type2_files))
print('Number of images in a train set of type 3: ', len(type3_files))
print('Total number of images in a train set: ', sum([len(type1_files), len(type2_files), len(type3_files)]))
Number of images in a train set of type 1:  1191
Number of images in a train set of type 2:  3567
Number of images in a train set of type 3:  1976
Total number of images in a train set:  6734
In [ ]:
# Building a dataframe mapping images and Cancer type

files_df = pd.DataFrame({
    'filename': type1_files + type2_files + type3_files,
    'label': ['Type_1'] * len(type1_files) + ['Type_2'] * len(type2_files) + ['Type_3'] * len(type3_files)
})

files_df
Out[ ]:
filename label
0 ../input/intel-mobileodt-cervical-cancer-scree... Type_1
1 ../input/intel-mobileodt-cervical-cancer-scree... Type_1
2 ../input/intel-mobileodt-cervical-cancer-scree... Type_1
3 ../input/intel-mobileodt-cervical-cancer-scree... Type_1
4 ../input/intel-mobileodt-cervical-cancer-scree... Type_1
... ... ...
6729 ../input/intel-mobileodt-cervical-cancer-scree... Type_3
6730 ../input/intel-mobileodt-cervical-cancer-scree... Type_3
6731 ../input/intel-mobileodt-cervical-cancer-scree... Type_3
6732 ../input/intel-mobileodt-cervical-cancer-scree... Type_3
6733 ../input/intel-mobileodt-cervical-cancer-scree... Type_3

6734 rows × 2 columns

In [ ]:
#Shuffle data

random_state = 42

files_df = files_df.sample(frac=1, random_state=random_state)
# files_df = files_df.sample(n=100, random_state=random_state)

files_df
Out[ ]:
filename label
381 ../input/intel-mobileodt-cervical-cancer-scree... Type_1
4696 ../input/intel-mobileodt-cervical-cancer-scree... Type_2
5577 ../input/intel-mobileodt-cervical-cancer-scree... Type_3
2018 ../input/intel-mobileodt-cervical-cancer-scree... Type_2
2185 ../input/intel-mobileodt-cervical-cancer-scree... Type_2
... ... ...
4863 ../input/intel-mobileodt-cervical-cancer-scree... Type_3
1446 ../input/intel-mobileodt-cervical-cancer-scree... Type_2
1188 ../input/intel-mobileodt-cervical-cancer-scree... Type_1
6471 ../input/intel-mobileodt-cervical-cancer-scree... Type_3
2153 ../input/intel-mobileodt-cervical-cancer-scree... Type_2

100 rows × 2 columns

Data exploration¶

In [ ]:
files_df.describe()
Out[ ]:
filename label
count 100 100
unique 100 3
top ../input/intel-mobileodt-cervical-cancer-scree... Type_2
freq 1 52
In [ ]:
#Check for duplicates
len(files_df[files_df.duplicated()])
Out[ ]:
0
In [ ]:
#Get count of each type 
type_count = pd.DataFrame(files_df['label'].value_counts())
type_count
Out[ ]:
count
label
Type_2 52
Type_3 34
Type_1 14
In [ ]:
print(list(type_count.columns)[0])
count
In [ ]:
# Display barplot of type count

plt.figure(figsize = (15, 6))
sns.barplot(x= type_count[list(type_count.columns)[0]], y= type_count.index.to_list())
plt.title('Cervical Cancer Type Distribution')
plt.grid(True)
plt.show()
In [ ]:
# Display sample images of types
for label in ('Type_1', 'Type_2', 'Type_3'):
    filepaths = files_df[files_df['label']==label]['filename'].values[:5]
    fig = plt.figure(figsize= (15, 6))
    for i, path in enumerate(filepaths):
        img = cv2.imread(path)
        img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
        img = cv2.resize(img, (224, 224))
        fig.add_subplot(1, 5, i+1)
        plt.imshow(img)
        plt.subplots_adjust(hspace=0.5)
        plt.axis(False)
        plt.title(label)

Data propocessing¶

In [ ]:
# Split training,val and test set : 70:15:15

train_files, test_files, train_labels, test_labels = train_test_split(files_df['filename'].values,
                                                                      files_df['label'].values, 
                                                                      test_size=0.3, 
                                                                      random_state=random_state)

test_files, val_files, test_labels, val_labels = train_test_split(test_files,
                                                                  test_labels, 
                                                                  test_size=0.5, 
                                                                  random_state=random_state)


print('Number of images in train set: ', train_files.shape)
print('Number of images in validation set: ', val_files.shape)
print('Number of images in test set: ', test_files.shape, '\n')

print('Train:', Counter(train_labels), '\nVal:', Counter(val_labels), '\nTest:', Counter(test_labels))
Number of images in train set:  (70,)
Number of images in validation set:  (15,)
Number of images in test set:  (15,) 

Train: Counter({'Type_2': 30, 'Type_3': 27, 'Type_1': 13}) 
Val: Counter({'Type_2': 12, 'Type_3': 2, 'Type_1': 1}) 
Test: Counter({'Type_2': 10, 'Type_3': 5})
In [ ]:
def load_images(files, labels):
    features = []
    correct_labels = []
    bad_images = 0
    
    for i in range(len(files)):
        try:
            img = cv2.imread(files[i])
            resized_img = cv2.resize(img, (160, 160))
            
            features.append(np.array(resized_img))
            correct_labels.append(labels[i])
                   
        except Exception as e:
            bad_images+=1
            print('Encoutered bad image')
    print('Bad images ecountered:', bad_images)
    return np.array(features), np.array(correct_labels)
In [ ]:
# Load training and evaluation data
train_features, train_labels = load_images(train_files, train_labels)
print('Train images loaded')

val_features, val_labels = load_images(val_files, val_labels)
print('Validation images loaded')

test_features, test_labels = load_images(test_files, test_labels)
print('test images loaded')
Bad images ecountered: 0
Train images loaded
Bad images ecountered: 0
Validation images loaded
Bad images ecountered: 0
test images loaded
In [ ]:
# check lengths of training and evaluation  sets
len(train_features), len(train_labels), len(val_features), len(val_labels), len(test_features), len(test_labels) 
Out[ ]:
(70, 70, 15, 15, 15, 15)
In [ ]:
BATCH_SIZE = 32
NUM_CLASSES = 3
EPOCHS = 10
INPUT_SHAPE = (160, 160, 3)
In [ ]:
# encode train+val sets text categories with labels
le = LabelEncoder()
le.fit(train_labels)

train_labels_enc = le.transform(train_labels)
val_labels_enc = le.transform(val_labels)

train_labels_1hotenc = tf.keras.utils.to_categorical(train_labels_enc, num_classes=NUM_CLASSES)
val_labels_1hotenc = tf.keras.utils.to_categorical(val_labels_enc, num_classes=NUM_CLASSES)

print(train_labels[:6], train_labels_enc[:6])
print(train_labels[:6], train_labels_1hotenc[:6])
['Type_3' 'Type_2' 'Type_2' 'Type_2' 'Type_2' 'Type_2'] [2 1 1 1 1 1]
['Type_3' 'Type_2' 'Type_2' 'Type_2' 'Type_2' 'Type_2'] [[0. 0. 1.]
 [0. 1. 0.]
 [0. 1. 0.]
 [0. 1. 0.]
 [0. 1. 0.]
 [0. 1. 0.]]
In [ ]:
le = LabelEncoder()
le.fit(test_labels)

test_labels_enc = le.transform(test_labels)

test_labels_1hotenc = tf.keras.utils.to_categorical(test_labels_enc, num_classes=NUM_CLASSES)


print(test_labels[:6], test_labels_enc[:6])
print(test_labels[:6], test_labels_1hotenc[:6])
['Type_3' 'Type_2' 'Type_2' 'Type_2' 'Type_3' 'Type_2'] [1 0 0 0 1 0]
['Type_3' 'Type_2' 'Type_2' 'Type_2' 'Type_3' 'Type_2'] [[0. 1. 0.]
 [1. 0. 0.]
 [1. 0. 0.]
 [1. 0. 0.]
 [0. 1. 0.]
 [1. 0. 0.]]

Data augmentation¶

In [ ]:
data_augmentation = tf.keras.Sequential([
  tf.keras.layers.RandomFlip('horizontal'),
  tf.keras.layers.RandomRotation(0.2),
])
In [ ]:
plt.figure(figsize=(10, 10))
first_image = train_features[0]
for i in range(9):
    ax = plt.subplot(3, 3, i + 1)
    augmented_image = data_augmentation(tf.expand_dims(first_image, 0))
    plt.imshow(augmented_image[0] / 255)
    plt.axis('off')
        

Metrics¶

In [ ]:
def get_accuracy_metrics(model, train_features=train_features, train_labels=train_labels_enc, test_features=test_features, test_labels=test_labels_enc, val_features=val_features, val_labels=val_labels_enc):    
    train_predicted = np.argmax(model.predict(train_features),axis=1)
    test_predicted = np.argmax(model.predict(test_features),axis=1)
    val_predicted = np.argmax(model.predict(val_features),axis=1)

    print("Train accuracy Score------------>")
    print ("{0:.3f}".format(accuracy_score(train_labels, train_predicted) *100), "%")
    
    print("Val accuracy Score--------->")
    print("{0:.3f}".format(accuracy_score(val_labels, val_predicted)*100), "%")
    
    print("Test accuracy Score--------->")
    print("{0:.3f}".format(accuracy_score(test_labels, test_predicted)*100), "%")
    
    print("F1 Score--------------->")
    print("{0:.3f}".format(f1_score(test_labels, test_predicted, average = 'weighted')*100), "%")
    
    print("Cohen Kappa Score------------->")
    print("{0:.3f}".format(cohen_kappa_score(test_labels, test_predicted)*100), "%")
    
    
    print("ROC AUC Score------------->")
    print("{0:.3f}".format(roc_auc_score(to_categorical(test_labels, num_classes = 3), test_predicted.reshape(-1, 1), multi_class='ovr')*100), "%")
    
    print("Recall-------------->")
    print("{0:.3f}".format(recall_score(test_labels, test_predicted, average = 'weighted')*100), "%")
    
    print("Precision-------------->")
    print("{0:.3f}".format(precision_score(test_labels, test_predicted, average = 'weighted')*100), "%")
    
    cf_matrix_test = confusion_matrix(test_labels, test_predicted)
    cf_matrix_val = confusion_matrix(val_labels, val_predicted)
    
    plt.figure(figsize = (12, 6))
    plt.subplot(121)
    sns.heatmap(cf_matrix_val, annot=True, cmap='Blues')
    plt.title("Val Confusion matrix")
    
    plt.subplot(122)
    sns.heatmap(cf_matrix_test, annot=True, cmap='Blues')
    plt.title("Test Confusion matrix")
    
    plt.show()

General Model Fit¶

In [ ]:
def learning_performance_chart(title, history):
    #plots a chart showing the change in accuracy and loss function over epochs
    f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))
    t = f.suptitle(title, fontsize=12)
    f.subplots_adjust(top=0.85, wspace=0.3)

    max_epoch = len(history.history['accuracy'])+1
    epoch_list = list(range(1,max_epoch))
    ax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')
    ax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')
    ax1.set_xticks(np.arange(1, max_epoch, 5))
    ax1.set_ylabel('Accuracy Value')
    ax1.set_xlabel('Epoch')
    ax1.set_title('Accuracy')
    l1 = ax1.legend(loc="best")

    ax2.plot(epoch_list, history.history['loss'], label='Train Loss')
    ax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')
    ax2.set_xticks(np.arange(1, max_epoch, 5))
    ax2.set_ylabel('Loss Value')
    ax2.set_xlabel('Epoch')
    ax2.set_title('Loss')
    l2 = ax2.legend(loc="best")

    
In [ ]:
def fit_model(model_name, base_model, train_features, train_labels, validate_it,training = False, epochs = EPOCHS, batch_size= BATCH_SIZE):
    
    inputs = tf.keras.Input(shape=INPUT_SHAPE)
    
    x = data_augmentation(inputs)
    x = base_model(x, training=training)
    
    if not model_name.startswith('CNN'):
        x = tf.keras.layers.GlobalAveragePooling2D()(x)
    
    x = tf.keras.layers.Dropout(0.2)(x)
    
    outputs = tf.keras.layers.Dense(3, activation='softmax')(x)
    
    model = tf.keras.Model(inputs, outputs)
    
    es = tf.keras.callbacks.EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=5)
    
    model.compile(loss='categorical_crossentropy', optimizer ='adam', metrics=['accuracy'])
    
    print("Model Summary.")
    
    print(model.summary())
    
    history = model.fit(x=train_features,y=train_labels ,validation_data=validate_it, epochs=epochs, batch_size=batch_size, verbose=1, callbacks=[es])

    learning_performance_chart(title="{} learning performance.".format(model_name), history=history)
    
    return model

CNN2¶

In [ ]:
model = tf.keras.Sequential([
 tf.keras.layers.BatchNormalization(),
 tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), padding = 'same', activation='relu'),
 tf.keras.layers.BatchNormalization(),
 tf.keras.layers.Conv2D(filters = 128, kernel_size = (3, 3), padding = 'same', activation='relu'),
 tf.keras.layers.Flatten(),
])

cnn2 = fit_model("CNN2", model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc), training=True)
Model Summary.
Model: "model_8"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_11 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 sequential_9 (Sequential)   (None, 3276800)           75916     
                                                                 
 dropout_15 (Dropout)        (None, 3276800)           0         
                                                                 
 dense_8 (Dense)             (None, 3)                 9830403   
                                                                 
=================================================================
Total params: 9,906,319
Trainable params: 9,906,185
Non-trainable params: 134
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 3s 904ms/step - loss: 231.7615 - accuracy: 0.4857 - val_loss: 108.2145 - val_accuracy: 0.8000
Epoch 2/10
3/3 [==============================] - 3s 880ms/step - loss: 178.6072 - accuracy: 0.4571 - val_loss: 703.4716 - val_accuracy: 0.0667
Epoch 3/10
3/3 [==============================] - 3s 930ms/step - loss: 362.9744 - accuracy: 0.1714 - val_loss: 79.9927 - val_accuracy: 0.4000
Epoch 4/10
3/3 [==============================] - 3s 892ms/step - loss: 95.4329 - accuracy: 0.5429 - val_loss: 243.1655 - val_accuracy: 0.2667
Epoch 5/10
3/3 [==============================] - 3s 785ms/step - loss: 165.0757 - accuracy: 0.4000 - val_loss: 482.3639 - val_accuracy: 0.2000
Epoch 6/10
3/3 [==============================] - 3s 762ms/step - loss: 202.6850 - accuracy: 0.5143 - val_loss: 107.9315 - val_accuracy: 0.8000
Epoch 7/10
3/3 [==============================] - 3s 808ms/step - loss: 397.2995 - accuracy: 0.4286 - val_loss: 61.4279 - val_accuracy: 0.5333
Epoch 8/10
3/3 [==============================] - 3s 720ms/step - loss: 179.2743 - accuracy: 0.3286 - val_loss: 338.7691 - val_accuracy: 0.2000
Epoch 9/10
3/3 [==============================] - 3s 791ms/step - loss: 135.3957 - accuracy: 0.4429 - val_loss: 36.5428 - val_accuracy: 0.5333
Epoch 10/10
3/3 [==============================] - 3s 876ms/step - loss: 70.9520 - accuracy: 0.5857 - val_loss: 140.9207 - val_accuracy: 0.2667
In [ ]:
print('CNN2 performance on the test set:')
get_accuracy_metrics(cnn2)
CNN2 performance on the test set:
3/3 [==============================] - 1s 195ms/step
1/1 [==============================] - 0s 169ms/step
1/1 [==============================] - 0s 169ms/step
Train accuracy Score------------>
41.429 %
Val accuracy Score--------->
26.667 %
Test accuracy Score--------->
53.333 %
F1 Score--------------->
51.852 %
Cohen Kappa Score------------->
-10.526 %
ROC AUC Score------------->
55.000 %
Recall-------------->
53.333 %
Precision-------------->
50.758 %

CNN3¶

In [ ]:
model = tf.keras.Sequential([
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.Conv2D(filters = 128, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters = 128, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.Flatten(),
    ])

cnn3 = fit_model("CNN3", model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc), training=True)
Model Summary.
Model: "model_9"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_12 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 sequential_10 (Sequential)  (None, 819200)            223500    
                                                                 
 dropout_17 (Dropout)        (None, 819200)            0         
                                                                 
 dense_9 (Dense)             (None, 3)                 2457603   
                                                                 
=================================================================
Total params: 2,681,103
Trainable params: 2,680,969
Non-trainable params: 134
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 4s 990ms/step - loss: 78.1532 - accuracy: 0.2286 - val_loss: 7.8706 - val_accuracy: 0.8000
Epoch 2/10
3/3 [==============================] - 3s 864ms/step - loss: 14.7362 - accuracy: 0.4286 - val_loss: 0.9287 - val_accuracy: 0.7333
Epoch 3/10
3/3 [==============================] - 3s 838ms/step - loss: 1.0454 - accuracy: 0.5143 - val_loss: 1.1029 - val_accuracy: 0.2000
Epoch 4/10
3/3 [==============================] - 3s 852ms/step - loss: 1.0944 - accuracy: 0.4857 - val_loss: 1.1000 - val_accuracy: 0.1333
Epoch 5/10
3/3 [==============================] - 3s 829ms/step - loss: 1.0970 - accuracy: 0.3857 - val_loss: 1.0989 - val_accuracy: 0.1333
Epoch 6/10
3/3 [==============================] - 3s 843ms/step - loss: 1.0978 - accuracy: 0.3857 - val_loss: 1.0992 - val_accuracy: 0.1333
Epoch 7/10
3/3 [==============================] - 3s 913ms/step - loss: 1.0971 - accuracy: 0.4000 - val_loss: 1.0988 - val_accuracy: 0.2000
Epoch 7: early stopping
In [ ]:
print('CNN3 performance on the test set:')
get_accuracy_metrics(cnn3)
CNN3 performance on the test set:
3/3 [==============================] - 1s 308ms/step
1/1 [==============================] - 0s 261ms/step
1/1 [==============================] - 0s 260ms/step
Train accuracy Score------------>
37.143 %
Val accuracy Score--------->
13.333 %
Test accuracy Score--------->
0.000 %
F1 Score--------------->
0.000 %
Cohen Kappa Score------------->
0.000 %
ROC AUC Score------------->
50.000 %
Recall-------------->
0.000 %
Precision-------------->
0.000 %
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

CNN4¶

In [ ]:
model = tf.keras.Sequential([
tf.keras.layers.BatchNormalization(),
tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), padding = 'same', activation='relu'),
tf.keras.layers.BatchNormalization(),
tf.keras.layers.Conv2D(filters = 128, kernel_size = (3, 3), padding = 'same', activation='relu'),
tf.keras.layers.MaxPooling2D(),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(filters = 128, kernel_size = (3, 3), padding = 'same', activation='relu'),
tf.keras.layers.MaxPooling2D(),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), padding = 'same', activation='relu'),
tf.keras.layers.Flatten(),
])

cnn4 = fit_model("CNN4", model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc), training=True)
Model Summary.
Model: "model_10"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_13 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 sequential_11 (Sequential)  (None, 102400)            297292    
                                                                 
 dropout_20 (Dropout)        (None, 102400)            0         
                                                                 
 dense_10 (Dense)            (None, 3)                 307203    
                                                                 
=================================================================
Total params: 604,495
Trainable params: 604,361
Non-trainable params: 134
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 4s 1s/step - loss: 10.6178 - accuracy: 0.4286 - val_loss: 1.0076 - val_accuracy: 0.6667
Epoch 2/10
3/3 [==============================] - 3s 962ms/step - loss: 1.2460 - accuracy: 0.4143 - val_loss: 2.4352 - val_accuracy: 0.1333
Epoch 3/10
3/3 [==============================] - 3s 893ms/step - loss: 1.1959 - accuracy: 0.4286 - val_loss: 0.8493 - val_accuracy: 0.8000
Epoch 4/10
3/3 [==============================] - 3s 868ms/step - loss: 1.0862 - accuracy: 0.4571 - val_loss: 0.6775 - val_accuracy: 0.8000
Epoch 5/10
3/3 [==============================] - 3s 890ms/step - loss: 1.1260 - accuracy: 0.4286 - val_loss: 0.9135 - val_accuracy: 0.6000
Epoch 6/10
3/3 [==============================] - 3s 905ms/step - loss: 1.0506 - accuracy: 0.4286 - val_loss: 1.0231 - val_accuracy: 0.2000
Epoch 7/10
3/3 [==============================] - 3s 915ms/step - loss: 1.0450 - accuracy: 0.4571 - val_loss: 1.0979 - val_accuracy: 0.2667
Epoch 8/10
3/3 [==============================] - 3s 930ms/step - loss: 1.0445 - accuracy: 0.5143 - val_loss: 1.0180 - val_accuracy: 0.5333
Epoch 9/10
3/3 [==============================] - 3s 916ms/step - loss: 0.9811 - accuracy: 0.5571 - val_loss: 0.7202 - val_accuracy: 0.6667
Epoch 9: early stopping
In [ ]:
print('CNN4 performance on the test set:')
get_accuracy_metrics(cnn4)
CNN4 performance on the test set:
3/3 [==============================] - 1s 269ms/step
1/1 [==============================] - 0s 219ms/step
1/1 [==============================] - 0s 273ms/step
Train accuracy Score------------>
47.143 %
Val accuracy Score--------->
66.667 %
Test accuracy Score--------->
33.333 %
F1 Score--------------->
18.519 %
Cohen Kappa Score------------->
6.250 %
ROC AUC Score------------->
60.000 %
Recall-------------->
33.333 %
Precision-------------->
12.821 %
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

CNN5¶

In [ ]:
model = tf.keras.Sequential([
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.Conv2D(filters=64, kernel_size=(
        3, 3), padding='same', activation='relu'),
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.Conv2D(filters=128, kernel_size=(
        3, 3), padding='same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters=128, kernel_size=(3, 3), padding='same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters=64, kernel_size=(
        3, 3), padding='same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters=32, kernel_size=(
        3, 3), padding='same', activation='relu'),
    tf.keras.layers.Flatten(),
])


cnn5 = fit_model("CNN5", model, train_features, train_labels_1hotenc,
                       (val_features, val_labels_1hotenc), training=True)
Model Summary.
Model: "model_11"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_14 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 sequential_12 (Sequential)  (None, 12800)             315756    
                                                                 
 dropout_24 (Dropout)        (None, 12800)             0         
                                                                 
 dense_11 (Dense)            (None, 3)                 38403     
                                                                 
=================================================================
Total params: 354,159
Trainable params: 354,025
Non-trainable params: 134
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 5s 988ms/step - loss: 2.8182 - accuracy: 0.3000 - val_loss: 0.8662 - val_accuracy: 0.6000
Epoch 2/10
3/3 [==============================] - 3s 916ms/step - loss: 1.2145 - accuracy: 0.3857 - val_loss: 0.9195 - val_accuracy: 0.6000
Epoch 3/10
3/3 [==============================] - 3s 940ms/step - loss: 1.1157 - accuracy: 0.4429 - val_loss: 0.8744 - val_accuracy: 0.7333
Epoch 4/10
3/3 [==============================] - 3s 1s/step - loss: 1.1492 - accuracy: 0.4429 - val_loss: 1.0782 - val_accuracy: 0.4667
Epoch 5/10
3/3 [==============================] - 4s 1s/step - loss: 1.0849 - accuracy: 0.4571 - val_loss: 1.1051 - val_accuracy: 0.4667
Epoch 6/10
3/3 [==============================] - 3s 984ms/step - loss: 1.0274 - accuracy: 0.4286 - val_loss: 1.0260 - val_accuracy: 0.2667
Epoch 6: early stopping
In [ ]:
print('CNN5 performance on the test set:')
get_accuracy_metrics(cnn5)
CNN5 performance on the test set:
3/3 [==============================] - 1s 305ms/step
1/1 [==============================] - 0s 268ms/step
1/1 [==============================] - 0s 277ms/step
Train accuracy Score------------>
52.857 %
Val accuracy Score--------->
33.333 %
Test accuracy Score--------->
6.667 %
F1 Score--------------->
9.524 %
Cohen Kappa Score------------->
2.326 %
ROC AUC Score------------->
55.000 %
Recall-------------->
6.667 %
Precision-------------->
16.667 %
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

CNN6¶

In [ ]:
model = tf.keras.Sequential([
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.Conv2D(filters = 128, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters = 128, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters = 32, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.Conv2D(filters = 32, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.Flatten(),
])

cnn6 = fit_model("CNN6", model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc), training=True)
Model Summary.
Model: "model_12"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_15 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 sequential_13 (Sequential)  (None, 12800)             325004    
                                                                 
 dropout_28 (Dropout)        (None, 12800)             0         
                                                                 
 dense_12 (Dense)            (None, 3)                 38403     
                                                                 
=================================================================
Total params: 363,407
Trainable params: 363,273
Non-trainable params: 134
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 4s 1s/step - loss: 1.7459 - accuracy: 0.3000 - val_loss: 0.7330 - val_accuracy: 0.7333
Epoch 2/10
3/3 [==============================] - 3s 1s/step - loss: 1.1575 - accuracy: 0.4286 - val_loss: 0.9248 - val_accuracy: 0.8667
Epoch 3/10
3/3 [==============================] - 3s 968ms/step - loss: 1.0844 - accuracy: 0.4571 - val_loss: 1.1701 - val_accuracy: 0.2000
Epoch 4/10
3/3 [==============================] - 3s 946ms/step - loss: 1.1349 - accuracy: 0.2714 - val_loss: 0.9220 - val_accuracy: 0.6667
Epoch 5/10
3/3 [==============================] - 3s 928ms/step - loss: 1.1647 - accuracy: 0.4571 - val_loss: 0.9900 - val_accuracy: 0.6000
Epoch 6/10
3/3 [==============================] - 3s 948ms/step - loss: 1.0425 - accuracy: 0.5429 - val_loss: 1.1288 - val_accuracy: 0.2000
Epoch 6: early stopping
In [ ]:
print('CNN6 performance on the test set:')
get_accuracy_metrics(cnn6)
CNN6 performance on the test set:
3/3 [==============================] - 1s 318ms/step
1/1 [==============================] - 0s 272ms/step
1/1 [==============================] - 0s 264ms/step
Train accuracy Score------------>
45.714 %
Val accuracy Score--------->
26.667 %
Test accuracy Score--------->
0.000 %
F1 Score--------------->
0.000 %
Cohen Kappa Score------------->
-2.273 %
ROC AUC Score------------->
45.000 %
Recall-------------->
0.000 %
Precision-------------->
0.000 %
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

CNN7¶

In [ ]:
model = tf.keras.Sequential([
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.Conv2D(filters = 128, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters = 128, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters = 32, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.Conv2D(filters = 32, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Conv2D(filters = 16, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.Flatten(),
])

cnn7 = fit_model("CNN7", model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc), training=True)
Model Summary.
Model: "model_13"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_16 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 sequential_14 (Sequential)  (None, 1600)              329628    
                                                                 
 dropout_32 (Dropout)        (None, 1600)              0         
                                                                 
 dense_13 (Dense)            (None, 3)                 4803      
                                                                 
=================================================================
Total params: 334,431
Trainable params: 334,297
Non-trainable params: 134
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 4s 957ms/step - loss: 1.6848 - accuracy: 0.3143 - val_loss: 1.1505 - val_accuracy: 0.2667
Epoch 2/10
3/3 [==============================] - 3s 937ms/step - loss: 1.1200 - accuracy: 0.4000 - val_loss: 0.8552 - val_accuracy: 0.8000
Epoch 3/10
3/3 [==============================] - 3s 934ms/step - loss: 1.1163 - accuracy: 0.4000 - val_loss: 0.9843 - val_accuracy: 0.5333
Epoch 4/10
3/3 [==============================] - 3s 926ms/step - loss: 1.0382 - accuracy: 0.4286 - val_loss: 1.0534 - val_accuracy: 0.3333
Epoch 5/10
3/3 [==============================] - 3s 938ms/step - loss: 1.0225 - accuracy: 0.4571 - val_loss: 0.9184 - val_accuracy: 0.5333
Epoch 6/10
3/3 [==============================] - 3s 954ms/step - loss: 0.9772 - accuracy: 0.5571 - val_loss: 0.8176 - val_accuracy: 0.6667
Epoch 7/10
3/3 [==============================] - 3s 953ms/step - loss: 0.9926 - accuracy: 0.5000 - val_loss: 1.2017 - val_accuracy: 0.3333
Epoch 8/10
3/3 [==============================] - 3s 933ms/step - loss: 1.0786 - accuracy: 0.4714 - val_loss: 1.2384 - val_accuracy: 0.1333
Epoch 9/10
3/3 [==============================] - 3s 890ms/step - loss: 1.0142 - accuracy: 0.4857 - val_loss: 1.0574 - val_accuracy: 0.4667
Epoch 10/10
3/3 [==============================] - 3s 913ms/step - loss: 0.9705 - accuracy: 0.5857 - val_loss: 0.8623 - val_accuracy: 0.7333
In [ ]:
print('CNN7 performance on the test set:')
get_accuracy_metrics(cnn7)
CNN7 performance on the test set:
3/3 [==============================] - 1s 308ms/step
1/1 [==============================] - 0s 265ms/step
1/1 [==============================] - 0s 272ms/step
Train accuracy Score------------>
52.857 %
Val accuracy Score--------->
66.667 %
Test accuracy Score--------->
13.333 %
F1 Score--------------->
8.889 %
Cohen Kappa Score------------->
-11.429 %
ROC AUC Score------------->
30.000 %
Recall-------------->
13.333 %
Precision-------------->
6.667 %
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

CNN8¶

In [ ]:
model = tf.keras.Sequential([
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.Conv2D(filters = 128, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters = 128, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters = 32, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.Conv2D(filters = 32, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Conv2D(filters = 16, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.Conv2D(filters = 16, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.Flatten(),
])

cnn8 = fit_model("CNN8", model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc), training=True)
Model Summary.
Model: "model_14"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_17 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 sequential_15 (Sequential)  (None, 1600)              331948    
                                                                 
 dropout_36 (Dropout)        (None, 1600)              0         
                                                                 
 dense_14 (Dense)            (None, 3)                 4803      
                                                                 
=================================================================
Total params: 336,751
Trainable params: 336,617
Non-trainable params: 134
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 4s 1s/step - loss: 1.1980 - accuracy: 0.3143 - val_loss: 0.9724 - val_accuracy: 0.5333
Epoch 2/10
3/3 [==============================] - 3s 967ms/step - loss: 1.0676 - accuracy: 0.4143 - val_loss: 1.0963 - val_accuracy: 0.4000
Epoch 3/10
3/3 [==============================] - 3s 920ms/step - loss: 1.0239 - accuracy: 0.4714 - val_loss: 0.9282 - val_accuracy: 0.4667
Epoch 4/10
3/3 [==============================] - 3s 922ms/step - loss: 1.0743 - accuracy: 0.4714 - val_loss: 0.8561 - val_accuracy: 0.4667
Epoch 5/10
3/3 [==============================] - 3s 929ms/step - loss: 1.0084 - accuracy: 0.4571 - val_loss: 0.9489 - val_accuracy: 0.6000
Epoch 6/10
3/3 [==============================] - 3s 936ms/step - loss: 1.0371 - accuracy: 0.4286 - val_loss: 0.9924 - val_accuracy: 0.3333
Epoch 7/10
3/3 [==============================] - 3s 932ms/step - loss: 0.9820 - accuracy: 0.5571 - val_loss: 0.8944 - val_accuracy: 0.5333
Epoch 8/10
3/3 [==============================] - 3s 918ms/step - loss: 0.9983 - accuracy: 0.4429 - val_loss: 0.7442 - val_accuracy: 0.7333
Epoch 9/10
3/3 [==============================] - 3s 910ms/step - loss: 0.9583 - accuracy: 0.4857 - val_loss: 0.9171 - val_accuracy: 0.6667
Epoch 10/10
3/3 [==============================] - 3s 916ms/step - loss: 0.9977 - accuracy: 0.4714 - val_loss: 1.0547 - val_accuracy: 0.6000
In [ ]:
print('CNN8 performance on the test set:')
get_accuracy_metrics(cnn8)
CNN8 performance on the test set:
3/3 [==============================] - 1s 317ms/step
1/1 [==============================] - 0s 258ms/step
1/1 [==============================] - 0s 267ms/step
Train accuracy Score------------>
61.429 %
Val accuracy Score--------->
40.000 %
Test accuracy Score--------->
13.333 %
F1 Score--------------->
13.333 %
Cohen Kappa Score------------->
2.500 %
ROC AUC Score------------->
55.000 %
Recall-------------->
13.333 %
Precision-------------->
13.333 %
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

ResNet 50¶

In [ ]:
base_model = tf.keras.applications.ResNet50(
    include_top=False,
    weights="imagenet",
    classes=NUM_CLASSES,
)

base_model.trainable = False

resnet50 = fit_model("ResNet50", base_model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc))
Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/resnet/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5
94765736/94765736 [==============================] - 11s 0us/step
Model Summary.
Model: "model_15"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_19 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 resnet50 (Functional)       (None, None, None, 2048)  23587712  
                                                                 
 global_average_pooling2d_2   (None, 2048)             0         
 (GlobalAveragePooling2D)                                        
                                                                 
 dropout_37 (Dropout)        (None, 2048)              0         
                                                                 
 dense_15 (Dense)            (None, 3)                 6147      
                                                                 
=================================================================
Total params: 23,593,859
Trainable params: 6,147
Non-trainable params: 23,587,712
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 3s 554ms/step - loss: 1.5936 - accuracy: 0.3857 - val_loss: 1.2789 - val_accuracy: 0.6667
Epoch 2/10
3/3 [==============================] - 1s 329ms/step - loss: 1.5735 - accuracy: 0.5000 - val_loss: 1.4265 - val_accuracy: 0.4667
Epoch 3/10
3/3 [==============================] - 1s 329ms/step - loss: 1.3143 - accuracy: 0.5571 - val_loss: 1.7017 - val_accuracy: 0.3333
Epoch 4/10
3/3 [==============================] - 1s 327ms/step - loss: 1.0376 - accuracy: 0.5143 - val_loss: 1.6307 - val_accuracy: 0.4000
Epoch 5/10
3/3 [==============================] - 1s 324ms/step - loss: 1.0204 - accuracy: 0.5857 - val_loss: 1.4383 - val_accuracy: 0.5333
Epoch 6/10
3/3 [==============================] - 1s 324ms/step - loss: 1.0220 - accuracy: 0.5143 - val_loss: 1.3037 - val_accuracy: 0.5333
Epoch 6: early stopping
In [ ]:
print('ResNet50 performance on the test set:')
get_accuracy_metrics(resnet50)
ResNet50 performance on the test set:
3/3 [==============================] - 1s 228ms/step
1/1 [==============================] - 0s 183ms/step
1/1 [==============================] - 0s 178ms/step
Train accuracy Score------------>
57.143 %
Val accuracy Score--------->
53.333 %
Test accuracy Score--------->
33.333 %
F1 Score--------------->
40.000 %
Cohen Kappa Score------------->
-7.143 %
ROC AUC Score------------->
44.000 %
Recall-------------->
33.333 %
Precision-------------->
51.111 %
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

VGG16¶

In [ ]:
base_model = tf.keras.applications.VGG16(
    include_top=False,
    weights="imagenet",
    input_shape=INPUT_SHAPE,
    classes=NUM_CLASSES,
    classifier_activation="softmax",
)

base_model.trainable = False
Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/vgg16/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5
58889256/58889256 [==============================] - 7s 0us/step
In [ ]:
vgg16 = fit_model("VGG16", base_model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc))
Model Summary.
Model: "model_16"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_21 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 vgg16 (Functional)          (None, 5, 5, 512)         14714688  
                                                                 
 global_average_pooling2d_3   (None, 512)              0         
 (GlobalAveragePooling2D)                                        
                                                                 
 dropout_38 (Dropout)        (None, 512)               0         
                                                                 
 dense_16 (Dense)            (None, 3)                 1539      
                                                                 
=================================================================
Total params: 14,716,227
Trainable params: 1,539
Non-trainable params: 14,714,688
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 3s 957ms/step - loss: 5.2476 - accuracy: 0.3857 - val_loss: 4.4457 - val_accuracy: 0.4667
Epoch 2/10
3/3 [==============================] - 3s 905ms/step - loss: 4.2855 - accuracy: 0.4429 - val_loss: 4.3819 - val_accuracy: 0.4667
Epoch 3/10
3/3 [==============================] - 3s 890ms/step - loss: 4.3707 - accuracy: 0.4571 - val_loss: 4.3926 - val_accuracy: 0.4667
Epoch 4/10
3/3 [==============================] - 3s 920ms/step - loss: 4.1395 - accuracy: 0.4714 - val_loss: 4.5551 - val_accuracy: 0.4667
Epoch 5/10
3/3 [==============================] - 3s 886ms/step - loss: 2.9570 - accuracy: 0.5429 - val_loss: 4.7753 - val_accuracy: 0.4667
Epoch 6/10
3/3 [==============================] - 3s 897ms/step - loss: 4.1382 - accuracy: 0.5286 - val_loss: 5.0092 - val_accuracy: 0.4000
Epoch 7/10
3/3 [==============================] - 3s 916ms/step - loss: 3.6442 - accuracy: 0.4857 - val_loss: 5.0746 - val_accuracy: 0.4000
Epoch 7: early stopping
In [ ]:
print('VGG16 performance on the test set:')
get_accuracy_metrics(vgg16)
VGG16 performance on the test set:
3/3 [==============================] - 2s 623ms/step
1/1 [==============================] - 1s 518ms/step
1/1 [==============================] - 1s 511ms/step
Train accuracy Score------------>
50.000 %
Val accuracy Score--------->
40.000 %
Test accuracy Score--------->
13.333 %
F1 Score--------------->
20.513 %
Cohen Kappa Score------------->
-2.632 %
ROC AUC Score------------->
46.000 %
Recall-------------->
13.333 %
Precision-------------->
44.444 %
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

VGG19¶

In [ ]:
base_model = tf.keras.applications.VGG19(
    include_top=False,
    weights="imagenet",
    input_shape=INPUT_SHAPE,
    classes=NUM_CLASSES,
    classifier_activation="softmax",
)

base_model.trainable = False
Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/vgg19/vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5
80134624/80134624 [==============================] - 8s 0us/step
In [ ]:
vgg19 = fit_model("VGG19", base_model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc))
Model Summary.
Model: "model_17"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_23 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 vgg19 (Functional)          (None, 5, 5, 512)         20024384  
                                                                 
 global_average_pooling2d_4   (None, 512)              0         
 (GlobalAveragePooling2D)                                        
                                                                 
 dropout_39 (Dropout)        (None, 512)               0         
                                                                 
 dense_17 (Dense)            (None, 3)                 1539      
                                                                 
=================================================================
Total params: 20,025,923
Trainable params: 1,539
Non-trainable params: 20,024,384
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 4s 1s/step - loss: 9.2581 - accuracy: 0.3000 - val_loss: 8.9689 - val_accuracy: 0.2000
Epoch 2/10
3/3 [==============================] - 3s 1s/step - loss: 6.6118 - accuracy: 0.3429 - val_loss: 6.0221 - val_accuracy: 0.3333
Epoch 3/10
3/3 [==============================] - 3s 1s/step - loss: 7.3799 - accuracy: 0.2714 - val_loss: 4.4366 - val_accuracy: 0.4000
Epoch 4/10
3/3 [==============================] - 3s 1s/step - loss: 6.4170 - accuracy: 0.3429 - val_loss: 3.6450 - val_accuracy: 0.4667
Epoch 5/10
3/3 [==============================] - 3s 1s/step - loss: 5.6530 - accuracy: 0.3143 - val_loss: 3.3127 - val_accuracy: 0.5333
Epoch 6/10
3/3 [==============================] - 3s 1s/step - loss: 5.5191 - accuracy: 0.3000 - val_loss: 2.9773 - val_accuracy: 0.4667
Epoch 7/10
3/3 [==============================] - 3s 1s/step - loss: 5.1576 - accuracy: 0.3857 - val_loss: 2.7574 - val_accuracy: 0.4000
Epoch 8/10
3/3 [==============================] - 3s 1s/step - loss: 4.2287 - accuracy: 0.4143 - val_loss: 2.8127 - val_accuracy: 0.4000
Epoch 9/10
3/3 [==============================] - 4s 1s/step - loss: 3.9558 - accuracy: 0.4714 - val_loss: 2.7563 - val_accuracy: 0.3333
Epoch 10/10
3/3 [==============================] - 3s 1s/step - loss: 4.3099 - accuracy: 0.3857 - val_loss: 2.8368 - val_accuracy: 0.3333
In [ ]:
print('VGG19 performance on the test set:')
get_accuracy_metrics(vgg19)
VGG19 performance on the test set:
3/3 [==============================] - 3s 791ms/step
1/1 [==============================] - 1s 608ms/step
1/1 [==============================] - 1s 636ms/step
Train accuracy Score------------>
45.714 %
Val accuracy Score--------->
33.333 %
Test accuracy Score--------->
26.667 %
F1 Score--------------->
35.556 %
Cohen Kappa Score------------->
8.333 %
ROC AUC Score------------->
41.000 %
Recall-------------->
26.667 %
Precision-------------->
80.000 %
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

MobileNet Pretranined¶

In [ ]:
base_model = tf.keras.applications.MobileNet(include_top=False, 
                                               weights='imagenet', 
                                               input_shape=INPUT_SHAPE)

base_model.trainable = False
Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/mobilenet/mobilenet_1_0_160_tf_no_top.h5
17225924/17225924 [==============================] - 2s 0us/step
In [ ]:
mobilenet = fit_model("MobileNet", base_model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc))
Model Summary.
Model: "model_18"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_25 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 mobilenet_1.00_160 (Functio  (None, 5, 5, 1024)       3228864   
 nal)                                                            
                                                                 
 global_average_pooling2d_5   (None, 1024)             0         
 (GlobalAveragePooling2D)                                        
                                                                 
 dropout_40 (Dropout)        (None, 1024)              0         
                                                                 
 dense_18 (Dense)            (None, 3)                 3075      
                                                                 
=================================================================
Total params: 3,231,939
Trainable params: 3,075
Non-trainable params: 3,228,864
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 1s 209ms/step - loss: 1.3250 - accuracy: 0.4429 - val_loss: 1.2073 - val_accuracy: 0.4000
Epoch 2/10
3/3 [==============================] - 0s 86ms/step - loss: 1.1244 - accuracy: 0.4286 - val_loss: 1.5010 - val_accuracy: 0.2000
Epoch 3/10
3/3 [==============================] - 0s 82ms/step - loss: 1.2593 - accuracy: 0.4286 - val_loss: 1.3026 - val_accuracy: 0.4667
Epoch 4/10
3/3 [==============================] - 0s 81ms/step - loss: 0.9933 - accuracy: 0.4857 - val_loss: 1.1075 - val_accuracy: 0.4667
Epoch 5/10
3/3 [==============================] - 0s 85ms/step - loss: 1.0599 - accuracy: 0.5000 - val_loss: 1.0483 - val_accuracy: 0.5333
Epoch 6/10
3/3 [==============================] - 0s 87ms/step - loss: 1.1146 - accuracy: 0.4429 - val_loss: 1.0646 - val_accuracy: 0.5333
Epoch 7/10
3/3 [==============================] - 0s 85ms/step - loss: 0.9108 - accuracy: 0.5571 - val_loss: 1.0995 - val_accuracy: 0.4667
Epoch 8/10
3/3 [==============================] - 0s 87ms/step - loss: 0.9597 - accuracy: 0.5286 - val_loss: 1.0835 - val_accuracy: 0.4667
Epoch 9/10
3/3 [==============================] - 0s 85ms/step - loss: 1.0522 - accuracy: 0.5000 - val_loss: 1.0310 - val_accuracy: 0.5333
Epoch 10/10
3/3 [==============================] - 0s 83ms/step - loss: 0.9976 - accuracy: 0.6000 - val_loss: 0.9536 - val_accuracy: 0.4667
In [ ]:
print('MobileNet performance on the test set:')
get_accuracy_metrics(mobilenet)
MobileNet performance on the test set:
3/3 [==============================] - 0s 51ms/step
1/1 [==============================] - 0s 49ms/step
1/1 [==============================] - 0s 51ms/step
Train accuracy Score------------>
55.714 %
Val accuracy Score--------->
46.667 %
Test accuracy Score--------->
26.667 %
F1 Score--------------->
37.436 %
Cohen Kappa Score------------->
2.941 %
ROC AUC Score------------->
22.000 %
Recall-------------->
26.667 %
Precision-------------->
73.333 %
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

MobileNet V2 pre-trained¶

In [ ]:
base_model = tf.keras.applications.MobileNetV2(include_top=False, 
                                               weights='imagenet', 
                                               input_shape=INPUT_SHAPE)

base_model.trainable = False
Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/mobilenet_v2/mobilenet_v2_weights_tf_dim_ordering_tf_kernels_1.0_160_no_top.h5
9406464/9406464 [==============================] - 1s 0us/step
In [ ]:
mobilenetv2 = fit_model("MobileNetV2", base_model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc))
Model Summary.
Model: "model_19"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_27 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 mobilenetv2_1.00_160 (Funct  (None, 5, 5, 1280)       2257984   
 ional)                                                          
                                                                 
 global_average_pooling2d_6   (None, 1280)             0         
 (GlobalAveragePooling2D)                                        
                                                                 
 dropout_41 (Dropout)        (None, 1280)              0         
                                                                 
 dense_19 (Dense)            (None, 3)                 3843      
                                                                 
=================================================================
Total params: 2,261,827
Trainable params: 3,843
Non-trainable params: 2,257,984
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 2s 304ms/step - loss: 1.8470 - accuracy: 0.2286 - val_loss: 1.1985 - val_accuracy: 0.4667
Epoch 2/10
3/3 [==============================] - 0s 82ms/step - loss: 1.4548 - accuracy: 0.3143 - val_loss: 0.8727 - val_accuracy: 0.6000
Epoch 3/10
3/3 [==============================] - 0s 85ms/step - loss: 1.3060 - accuracy: 0.4143 - val_loss: 1.0550 - val_accuracy: 0.4000
Epoch 4/10
3/3 [==============================] - 0s 82ms/step - loss: 1.1841 - accuracy: 0.4714 - val_loss: 1.4036 - val_accuracy: 0.4000
Epoch 5/10
3/3 [==============================] - 0s 89ms/step - loss: 1.1979 - accuracy: 0.4143 - val_loss: 1.3764 - val_accuracy: 0.3333
Epoch 6/10
3/3 [==============================] - 0s 79ms/step - loss: 1.2333 - accuracy: 0.4143 - val_loss: 1.1880 - val_accuracy: 0.4000
Epoch 7/10
3/3 [==============================] - 0s 81ms/step - loss: 1.1137 - accuracy: 0.4714 - val_loss: 1.1701 - val_accuracy: 0.4000
Epoch 7: early stopping
In [ ]:
print('MobileNetV2 performance on the test set:')
get_accuracy_metrics(mobilenetv2)
MobileNetV2 performance on the test set:
3/3 [==============================] - 1s 53ms/step
1/1 [==============================] - 0s 47ms/step
1/1 [==============================] - 0s 45ms/step
Train accuracy Score------------>
50.000 %
Val accuracy Score--------->
40.000 %
Test accuracy Score--------->
6.667 %
F1 Score--------------->
6.061 %
Cohen Kappa Score------------->
-13.514 %
ROC AUC Score------------->
50.000 %
Recall-------------->
6.667 %
Precision-------------->
5.556 %
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

Inception V3¶

In [ ]:
base_model = tf.keras.applications.InceptionV3(
    include_top=False,
    weights="imagenet",
    input_shape=INPUT_SHAPE,
    classes=NUM_CLASSES,
    classifier_activation="softmax",
)

base_model.trainable = False

incpetionv3 = fit_model("InceptionV3", base_model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc))
Model Summary.
Model: "model_20"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_29 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 inception_v3 (Functional)   (None, 3, 3, 2048)        21802784  
                                                                 
 global_average_pooling2d_7   (None, 2048)             0         
 (GlobalAveragePooling2D)                                        
                                                                 
 dropout_42 (Dropout)        (None, 2048)              0         
                                                                 
 dense_20 (Dense)            (None, 3)                 6147      
                                                                 
=================================================================
Total params: 21,808,931
Trainable params: 6,147
Non-trainable params: 21,802,784
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 3s 443ms/step - loss: 50.7983 - accuracy: 0.2857 - val_loss: 6.4663 - val_accuracy: 0.6000
Epoch 2/10
3/3 [==============================] - 0s 151ms/step - loss: 27.2885 - accuracy: 0.4714 - val_loss: 9.8325 - val_accuracy: 0.8000
Epoch 3/10
3/3 [==============================] - 0s 144ms/step - loss: 30.6071 - accuracy: 0.4714 - val_loss: 7.0202 - val_accuracy: 0.6000
Epoch 4/10
3/3 [==============================] - 0s 143ms/step - loss: 21.1291 - accuracy: 0.5000 - val_loss: 6.0973 - val_accuracy: 0.6000
Epoch 5/10
3/3 [==============================] - 0s 144ms/step - loss: 19.5554 - accuracy: 0.3714 - val_loss: 14.1759 - val_accuracy: 0.4000
Epoch 6/10
3/3 [==============================] - 0s 142ms/step - loss: 25.3541 - accuracy: 0.3286 - val_loss: 3.9975 - val_accuracy: 0.6000
Epoch 7/10
3/3 [==============================] - 0s 144ms/step - loss: 15.8824 - accuracy: 0.5143 - val_loss: 1.7173 - val_accuracy: 0.8667
Epoch 8/10
3/3 [==============================] - 0s 144ms/step - loss: 21.8860 - accuracy: 0.4286 - val_loss: 1.7265 - val_accuracy: 0.7333
Epoch 9/10
3/3 [==============================] - 0s 145ms/step - loss: 17.9414 - accuracy: 0.5286 - val_loss: 1.0400 - val_accuracy: 0.7333
Epoch 10/10
3/3 [==============================] - 0s 142ms/step - loss: 19.0953 - accuracy: 0.4286 - val_loss: 1.6771 - val_accuracy: 0.8000
In [ ]:
print('InceptionV3 performance on the test set:')
get_accuracy_metrics(incpetionv3)
InceptionV3 performance on the test set:
3/3 [==============================] - 1s 93ms/step
1/1 [==============================] - 0s 83ms/step
1/1 [==============================] - 0s 83ms/step
Train accuracy Score------------>
61.429 %
Val accuracy Score--------->
80.000 %
Test accuracy Score--------->
26.667 %
F1 Score--------------->
25.455 %
Cohen Kappa Score------------->
0.000 %
ROC AUC Score------------->
37.000 %
Recall-------------->
26.667 %
Precision-------------->
76.667 %
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

DenseNet 121¶

In [ ]:
base_model = tf.keras.applications.DenseNet121(
    include_top=False,
    weights="imagenet",
    input_shape=INPUT_SHAPE,
    classes=NUM_CLASSES,
    classifier_activation="softmax",
)

base_model.trainable = False

densenet121 = fit_model("DenseNet121", base_model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc))
Model Summary.
Model: "model_21"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_31 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 densenet121 (Functional)    (None, 5, 5, 1024)        7037504   
                                                                 
 global_average_pooling2d_8   (None, 1024)             0         
 (GlobalAveragePooling2D)                                        
                                                                 
 dropout_43 (Dropout)        (None, 1024)              0         
                                                                 
 dense_21 (Dense)            (None, 3)                 3075      
                                                                 
=================================================================
Total params: 7,040,579
Trainable params: 3,075
Non-trainable params: 7,037,504
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 5s 688ms/step - loss: 8.9645 - accuracy: 0.2571 - val_loss: 3.7331 - val_accuracy: 0.1333
Epoch 2/10
3/3 [==============================] - 1s 296ms/step - loss: 5.6826 - accuracy: 0.3571 - val_loss: 3.8929 - val_accuracy: 0.1333
Epoch 3/10
3/3 [==============================] - 1s 282ms/step - loss: 5.4314 - accuracy: 0.4143 - val_loss: 2.9998 - val_accuracy: 0.3333
Epoch 4/10
3/3 [==============================] - 1s 283ms/step - loss: 5.0640 - accuracy: 0.4857 - val_loss: 2.8958 - val_accuracy: 0.3333
Epoch 5/10
3/3 [==============================] - 1s 278ms/step - loss: 4.2808 - accuracy: 0.5143 - val_loss: 3.0649 - val_accuracy: 0.3333
Epoch 6/10
3/3 [==============================] - 1s 283ms/step - loss: 5.0579 - accuracy: 0.4714 - val_loss: 3.3136 - val_accuracy: 0.2667
Epoch 7/10
3/3 [==============================] - 1s 286ms/step - loss: 3.3413 - accuracy: 0.5429 - val_loss: 4.1449 - val_accuracy: 0.2667
Epoch 8/10
3/3 [==============================] - 1s 278ms/step - loss: 4.7983 - accuracy: 0.4571 - val_loss: 4.1632 - val_accuracy: 0.2667
Epoch 9/10
3/3 [==============================] - 1s 277ms/step - loss: 3.8919 - accuracy: 0.4571 - val_loss: 3.7137 - val_accuracy: 0.2667
Epoch 9: early stopping
In [ ]:
print('DenseNet121 performance on the test set:')
get_accuracy_metrics(densenet121)
DenseNet121 performance on the test set:
3/3 [==============================] - 1s 191ms/step
1/1 [==============================] - 0s 158ms/step
1/1 [==============================] - 0s 163ms/step
Train accuracy Score------------>
55.714 %
Val accuracy Score--------->
26.667 %
Test accuracy Score--------->
26.667 %
F1 Score--------------->
38.889 %
Cohen Kappa Score------------->
13.158 %
ROC AUC Score------------->
51.000 %
Recall-------------->
26.667 %
Precision-------------->
88.889 %
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

DenseNet 169¶

In [ ]:
base_model = tf.keras.applications.DenseNet169(
    include_top=False,
    weights="imagenet",
    input_shape=INPUT_SHAPE,
    classes=NUM_CLASSES,
    classifier_activation="softmax",
)

base_model.trainable = False

densenet169 = fit_model("DenseNet169", base_model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc))
Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/densenet/densenet169_weights_tf_dim_ordering_tf_kernels_notop.h5
51877672/51877672 [==============================] - 5s 0us/step
Model Summary.
Model: "model_22"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_33 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 densenet169 (Functional)    (None, 5, 5, 1664)        12642880  
                                                                 
 global_average_pooling2d_9   (None, 1664)             0         
 (GlobalAveragePooling2D)                                        
                                                                 
 dropout_44 (Dropout)        (None, 1664)              0         
                                                                 
 dense_22 (Dense)            (None, 3)                 4995      
                                                                 
=================================================================
Total params: 12,647,875
Trainable params: 4,995
Non-trainable params: 12,642,880
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 6s 934ms/step - loss: 12.7267 - accuracy: 0.2714 - val_loss: 4.5255 - val_accuracy: 0.4667
Epoch 2/10
3/3 [==============================] - 1s 340ms/step - loss: 8.7353 - accuracy: 0.4571 - val_loss: 3.0815 - val_accuracy: 0.6667
Epoch 3/10
3/3 [==============================] - 1s 340ms/step - loss: 11.5507 - accuracy: 0.3429 - val_loss: 2.8783 - val_accuracy: 0.6667
Epoch 4/10
3/3 [==============================] - 1s 344ms/step - loss: 7.1117 - accuracy: 0.4714 - val_loss: 3.5252 - val_accuracy: 0.4000
Epoch 5/10
3/3 [==============================] - 1s 337ms/step - loss: 6.5601 - accuracy: 0.4000 - val_loss: 7.6735 - val_accuracy: 0.2000
Epoch 6/10
3/3 [==============================] - 1s 343ms/step - loss: 6.8822 - accuracy: 0.3714 - val_loss: 7.9800 - val_accuracy: 0.1333
Epoch 7/10
3/3 [==============================] - 1s 336ms/step - loss: 6.1544 - accuracy: 0.3857 - val_loss: 5.3288 - val_accuracy: 0.3333
Epoch 8/10
3/3 [==============================] - 1s 333ms/step - loss: 5.3028 - accuracy: 0.4571 - val_loss: 3.8081 - val_accuracy: 0.4000
Epoch 8: early stopping
In [ ]:
print('DenseNet169 performance on the test set:')
get_accuracy_metrics(densenet169)
DenseNet169 performance on the test set:
3/3 [==============================] - 2s 233ms/step
1/1 [==============================] - 0s 193ms/step
1/1 [==============================] - 0s 187ms/step
Train accuracy Score------------>
47.143 %
Val accuracy Score--------->
40.000 %
Test accuracy Score--------->
20.000 %
F1 Score--------------->
24.242 %
Cohen Kappa Score------------->
2.703 %
ROC AUC Score------------->
43.000 %
Recall-------------->
20.000 %
Precision-------------->
77.778 %
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

InceptionResNetv2¶

In [ ]:
base_model = tf.keras.applications.InceptionResNetV2(
    include_top=False,
    weights="imagenet",
    input_shape=INPUT_SHAPE,
    classes=NUM_CLASSES,
    classifier_activation="softmax",
)

base_model.trainable = False

inceptionresnetv2 = fit_model("InceptionResNetV2", base_model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc))
Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/inception_resnet_v2/inception_resnet_v2_weights_tf_dim_ordering_tf_kernels_notop.h5
219055592/219055592 [==============================] - 26s 0us/step
Model Summary.
Model: "model_23"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_35 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 inception_resnet_v2 (Functi  (None, 3, 3, 1536)       54336736  
 onal)                                                           
                                                                 
 global_average_pooling2d_10  (None, 1536)             0         
  (GlobalAveragePooling2D)                                       
                                                                 
 dropout_45 (Dropout)        (None, 1536)              0         
                                                                 
 dense_23 (Dense)            (None, 3)                 4611      
                                                                 
=================================================================
Total params: 54,341,347
Trainable params: 4,611
Non-trainable params: 54,336,736
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 7s 1s/step - loss: 153.0160 - accuracy: 0.2857 - val_loss: 102.0554 - val_accuracy: 0.1333
Epoch 2/10
3/3 [==============================] - 1s 356ms/step - loss: 154.2074 - accuracy: 0.4714 - val_loss: 47.5271 - val_accuracy: 0.8000
Epoch 3/10
3/3 [==============================] - 1s 365ms/step - loss: 144.2635 - accuracy: 0.4571 - val_loss: 89.0682 - val_accuracy: 0.1333
Epoch 4/10
3/3 [==============================] - 1s 351ms/step - loss: 137.2067 - accuracy: 0.4143 - val_loss: 38.8472 - val_accuracy: 0.2000
Epoch 5/10
3/3 [==============================] - 1s 359ms/step - loss: 90.6458 - accuracy: 0.4286 - val_loss: 32.4482 - val_accuracy: 0.8000
Epoch 6/10
3/3 [==============================] - 1s 355ms/step - loss: 128.8864 - accuracy: 0.3429 - val_loss: 35.8594 - val_accuracy: 0.8000
Epoch 7/10
3/3 [==============================] - 1s 353ms/step - loss: 102.2348 - accuracy: 0.3714 - val_loss: 14.2647 - val_accuracy: 0.7333
Epoch 8/10
3/3 [==============================] - 1s 358ms/step - loss: 106.2678 - accuracy: 0.3857 - val_loss: 120.1697 - val_accuracy: 0.1333
Epoch 9/10
3/3 [==============================] - 1s 359ms/step - loss: 153.3878 - accuracy: 0.3571 - val_loss: 20.0372 - val_accuracy: 0.1333
Epoch 10/10
3/3 [==============================] - 1s 358ms/step - loss: 104.0266 - accuracy: 0.4000 - val_loss: 31.8369 - val_accuracy: 0.8000
In [ ]:
print('InceptionResNetV2 performance on the test set:')
get_accuracy_metrics(inceptionresnetv2)
InceptionResNetV2 performance on the test set:
3/3 [==============================] - 2s 240ms/step
1/1 [==============================] - 0s 198ms/step
1/1 [==============================] - 0s 206ms/step
Train accuracy Score------------>
42.857 %
Val accuracy Score--------->
80.000 %
Test accuracy Score--------->
33.333 %
F1 Score--------------->
16.667 %
Cohen Kappa Score------------->
0.000 %
ROC AUC Score------------->
50.000 %
Recall-------------->
33.333 %
Precision-------------->
11.111 %
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

ResNet101¶

In [ ]:
base_model = tf.keras.applications.ResNet101(
    include_top=False,
    weights="imagenet",
    input_shape=INPUT_SHAPE,
    classes=NUM_CLASSES,
    classifier_activation="softmax",
)

base_model.trainable = False

resnet101 = fit_model("ResNet101", base_model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc))
Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/resnet/resnet101_weights_tf_dim_ordering_tf_kernels_notop.h5
171446536/171446536 [==============================] - 19s 0us/step
Model Summary.
Model: "model_24"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_37 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 resnet101 (Functional)      (None, 5, 5, 2048)        42658176  
                                                                 
 global_average_pooling2d_11  (None, 2048)             0         
  (GlobalAveragePooling2D)                                       
                                                                 
 dropout_46 (Dropout)        (None, 2048)              0         
                                                                 
 dense_24 (Dense)            (None, 3)                 6147      
                                                                 
=================================================================
Total params: 42,664,323
Trainable params: 6,147
Non-trainable params: 42,658,176
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 6s 1s/step - loss: 1.4095 - accuracy: 0.4571 - val_loss: 1.0700 - val_accuracy: 0.6000
Epoch 2/10
3/3 [==============================] - 2s 574ms/step - loss: 1.0785 - accuracy: 0.5714 - val_loss: 1.1273 - val_accuracy: 0.5333
Epoch 3/10
3/3 [==============================] - 2s 576ms/step - loss: 0.9201 - accuracy: 0.6286 - val_loss: 1.6148 - val_accuracy: 0.4000
Epoch 4/10
3/3 [==============================] - 2s 578ms/step - loss: 0.9295 - accuracy: 0.6143 - val_loss: 1.6263 - val_accuracy: 0.4000
Epoch 5/10
3/3 [==============================] - 2s 577ms/step - loss: 1.0989 - accuracy: 0.6429 - val_loss: 1.3021 - val_accuracy: 0.4667
Epoch 6/10
3/3 [==============================] - 2s 580ms/step - loss: 0.9922 - accuracy: 0.5857 - val_loss: 1.2924 - val_accuracy: 0.4000
Epoch 6: early stopping
In [ ]:
print('ResNet101 performance on the test set:')
get_accuracy_metrics(resnet101)
ResNet101 performance on the test set:
3/3 [==============================] - 2s 411ms/step
1/1 [==============================] - 0s 321ms/step
1/1 [==============================] - 0s 322ms/step
Train accuracy Score------------>
65.714 %
Val accuracy Score--------->
40.000 %
Test accuracy Score--------->
13.333 %
F1 Score--------------->
15.812 %
Cohen Kappa Score------------->
-21.875 %
ROC AUC Score------------->
56.000 %
Recall-------------->
13.333 %
Precision-------------->
26.984 %
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))